#!/usr/bin/env python
# -*- coding:utf-8 -*-

'''
@Author: Vulkey_Chen (admin@gh0st.cn)
@Blog: https://gh0st.cn
@Data: 2019-04-25
@Team: Mystery Security Team (MSTSEC)
@Function: get top domain && generate url && url filter
'''

import requests, re, urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# get top level domain
def getDomain(domain):
    res = domain
    domains = ["aaa",".abudhabi",".ba",".porn",".post",".pr",".pramerica",".praxi",".press",".prime",".pro",".prod",".productions",".baby",".prof",".progressive",".promo",".properties",".property",".protection",".pru",".prudential",".ps",".pt",".baidu",".pub",".pw",".pwc",".py",".qa",".qpon",".quebec",".quest",".qvc",".racing",".banamex",".radio",".raid",".re",".read",".realestate",".realtor",".realty",".recipes",".red",".redstone",".bananarepublic",".redumbrella",".rehab",".reise",".reisen",".reit",".reliance",".ren",".rent",".rentals",".repair",".band",".report",".republican",".rest",".restaurant",".review",".reviews",".rexroth",".rich",".richardli",".ricoh",".bank",".rightathome",".ril",".rio",".rip",".rmit",".ro",".rocher",".rocks",".rodeo",".rogers",".bar",".room",".rs",".rsvp",".ru",".rugby",".ruhr",".run",".rw",".rwe",".ryukyu",".barcelona",".sa",".saarland",".safe",".safety",".sakura",".sale",".salon",".samsclub",".samsung",".sandvik",".barclaycard",".sandvikcoromant",".sanofi",".sap",".sapo",".sarl",".sas",".save",".saxo",".sb",".sbi",".ac",".barclays",".sbs",".sc",".sca",".scb",".schaeffler",".schmidt",".scholarships",".school",".schule",".schwarz",".barefoot",".science",".scjohnson",".scor",".scot",".sd",".se",".search",".seat",".secure",".security",".bargains",".seek",".select",".sener",".services",".ses",".seven",".sew",".sex",".sexy",".sfr",".baseball",".sg",".sh",".shangrila",".sharp",".shaw",".shell",".shia",".shiksha",".shoes",".shop",".basketball",".shopping",".shouji",".show",".showtime",".shriram",".si",".silk",".sina",".singles",".site",".bauhaus",".sj",".sk",".ski",".skin",".sky",".skype",".sl",".sling",".sm",".smart",".bayern",".smile",".sn",".sncf",".so",".soccer",".social",".softbank",".software",".sohu",".solar",".bb",".solutions",".song",".sony",".soy",".space",".spiegel",".sport",".spot",".spreadbetting",".sr",".bbc",".srl",".srt",".ss",".st",".stada",".staples",".star",".starhub",".statebank",".statefarm",".bbt",".statoil",".stc",".stcgroup",".stockholm",".storage",".store",".stream",".studio",".study",".style",".academy",".bbva",".su",".sucks",".supplies",".supply",".support",".surf",".surgery",".suzuki",".sv",".swatch",".bcg",".swiftcover",".swiss",".sx",".sy",".sydney",".symantec",".systems",".sz",".tab",".taipei",".bcn",".talk",".taobao",".target",".tatamotors",".tatar",".tattoo",".tax",".taxi",".tc",".tci",".bd",".td",".tdk",".team",".tech",".technology",".tel",".telecity",".telefonica",".temasek",".tennis",".be",".teva",".tf",".tg",".th",".thd",".theater",".theatre",".tiaa",".tickets",".tienda",".beats",".tiffany",".tips",".tires",".tirol",".tj",".tjmaxx",".tjx",".tk",".tkmaxx",".tl",".beauty",".tm",".tmall",".tn",".to",".today",".tokyo",".tools",".top",".toray",".toshiba",".beer",".total",".tours",".town",".toyota",".toys",".tp",".tr",".trade",".trading",".training",".bentley",".travel",".travelchannel",".travelers",".travelersinsurance",".trust",".trv",".tt",".tube",".tui",".tunes",".berlin",".tushu",".tv",".tvs",".tw",".tz",".ua",".ubank",".ubs",".uconnect",".ug",".accenture",".best",".uk",".um",".unicom",".university",".uno",".uol",".ups",".us",".uy",".uz",".bestbuy",".va",".vacations",".vana",".vanguard",".vc",".ve",".vegas",".ventures",".verisign",".vermögensberater",".bet",".vermögensberatung",".versicherung",".vet",".vg",".vi",".viajes",".video",".vig",".viking",".villas",".bf",".vin",".vip",".virgin",".visa",".vision",".vista",".vistaprint",".viva",".vivo",".vlaanderen",".bg",".vn",".vodka",".volkswagen",".volvo",".vote",".voting",".voto",".voyage",".vu",".vuelos",".bh",".wales",".walmart",".walter",".wang",".wanggou",".warman",".watch",".watches",".weather",".weatherchannel",".bharti",".webcam",".weber",".website",".wed",".wedding",".weibo",".weir",".wf",".whoswho",".wien",".bi",".wiki",".williamhill",".win",".windows",".wine",".winners",".wme",".wolterskluwer",".woodside",".work",".bible",".works",".world",".wow",".ws",".wtc",".wtf",".xbox",".xerox",".xfinity",".xihuan",".bid",".xin",".xperia",".xxx",".xyz",".yachts",".yahoo",".yamaxun",".yandex",".ye",".yodobashi",".accountant",".bike",".yoga",".yokohama",".you",".youtube",".yt",".yun",".za",".zappos",".zara",".zero",".bing",".zip",".zippo",".zm",".zone",".zuerich",".zw",".bingo",".bio",".biz",".bj",".bl",".black",".blackfriday",".blanco",".accountants",".blockbuster",".bloomberg",".blue",".bm",".bms",".руc",".bn",".bnl",".bnpparibas",".aco",".bo",".boats",".boehringer",".bofa",".bom",".bond",".boo",".book",".booking",".boots",".active",".bosch",".bostik",".boston",".bot",".boutique",".box",".bq",".br",".bradesco",".bridgestone",".actor",".broadway",".broker",".brother",".brussels",".bs",".bt",".budapest",".bugatti",".build",".builders",".ad",".business",".buy",".buzz",".bv",".bw",".by",".bz",".bzh",".ca",".cab",".aarp",".adac",".cafe",".cal",".call",".calvinklein",".cam",".camera",".camp",".cancerresearch",".canon",".capetown",".ads",".capital",".capitalone",".car",".caravan",".cards",".care",".career",".careers",".cars",".cartier",".adult",".casa",".case",".caseih",".cash",".casino",".cat",".catering",".catholic",".cba",".cbn",".ae",".cbre",".cbs",".cc",".cd",".ceb",".center",".ceo",".cern",".cf",".cfa",".aeg",".cfd",".cg",".ch",".chanel",".channel",".chase",".chat",".cheap",".chintai",".chloe",".aero",".christmas",".chrome",".chrysler",".church",".ci",".cipriani",".circle",".cisco",".citadel",".citi",".aetna",".citic",".city",".cityeats",".ck",".cl",".claims",".cleaning",".click",".clinic",".clinique",".af",".clothing",".cloud",".club",".clubmed",".cm",".cn",".co",".coach",".codes",".coffee",".afamilycompany",".college",".cologne",".com",".comcast",".commbank",".community",".company",".compare",".computer",".comsec",".afl",".condos",".construction",".consulting",".contact",".contractors",".cooking",".cookingchannel",".cool",".coop",".corsica",".abarth",".africa",".country",".coupon",".coupons",".courses",".cr",".credit",".creditcard",".creditunion",".cricket",".crown",".ag",".crs",".cruise",".cruises",".csc",".cu",".cuisinella",".cv",".cw",".cx",".cy",".agakhan",".cymru",".cyou",".cz",".dabur",".dad",".dance",".data",".date",".dating",".datsun",".agency",".day",".dclk",".dds",".de",".deal",".dealer",".deals",".degree",".delivery",".dell",".ai",".deloitte",".delta",".democrat",".dental",".dentist",".desi",".design",".dev",".dhl",".diamonds",".aig",".diet",".digital",".direct",".directory",".discount",".discover",".dish",".diy",".dj",".dk",".aigo",".dm",".dnp",".do",".docs",".doctor",".dodge",".dog",".doha",".domains",".doosan",".airbus",".dot",".download",".drive",".dtv",".dubai",".duck",".dunlop",".duns",".dupont",".durban",".airforce",".dvag",".dvr",".dz",".earth",".eat",".ec",".eco",".edeka",".edu",".education",".airtel",".ee",".eg",".eh",".email",".emerck",".energy",".engineer",".engineering",".enterprises",".epost",".abb",".akdn",".epson",".equipment",".er",".ericsson",".erni",".es",".esq",".estate",".esurance",".et",".al",".etisalat",".eu",".eurovision",".eus",".events",".everbank",".exchange",".expert",".exposed",".express",".alfaromeo",".extraspace",".fage",".fail",".fairwinds",".faith",".family",".fan",".fans",".farm",".farmers",".alibaba",".fashion",".fast",".fedex",".feedback",".ferrari",".ferrero",".fi",".fiat",".fidelity",".fido",".alipay",".film",".final",".finance",".financial",".fire",".firestone",".firmdale",".fish",".fishing",".fit",".allfinanz",".fitness",".fj",".fk",".flickr",".flights",".flir",".florist",".flowers",".flsmidth",".fly",".allstate",".fm",".fo",".foo",".food",".foodnetwork",".football",".ford",".forex",".forsale",".forum",".ally",".foundation",".fox",".fr",".free",".fresenius",".frl",".frogans",".frontdoor",".frontier",".ftr",".alsace",".fujitsu",".fujixerox",".fun",".fund",".furniture",".futbol",".fyi",".ga",".gal",".gallery",".alstom",".gallo",".gallup",".game",".games",".gap",".garden",".gb",".gbiz",".gd",".gdn",".abbott",".am",".ge",".gea",".gent",".genting",".george",".gf",".gg",".ggee",".gh",".gi",".americanexpress",".gift",".gifts",".gives",".giving",".gl",".glade",".glass",".gle",".global",".globo",".americanfamily",".gm",".gmail",".gmbh",".gmo",".gmx",".gn",".godaddy",".gold",".goldpoint",".golf",".amex",".goo",".goodhands",".goodyear",".goog",".google",".gop",".got",".gov",".gp",".gq",".amfam",".gr",".grainger",".graphics",".gratis",".green",".gripe",".grocery",".group",".gs",".gt",".amica",".gu",".guardian",".gucci",".guge",".guide",".guitars",".guru",".gw",".gy",".hair",".amsterdam",".hamburg",".hangout",".haus",".hbo",".hdfc",".hdfcbank",".health",".healthcare",".help",".helsinki",".an",".here",".hermes",".hgtv",".hiphop",".hisamitsu",".hitachi",".hiv",".hk",".hkt",".hm",".analytics",".hn",".hockey",".holdings",".holiday",".homedepot",".homegoods",".homes",".homesense",".honda",".honeywell",".android",".horse",".hospital",".host",".hosting",".hot",".hoteles",".hotels",".hotmail",".house",".how",".abbvie",".anquan",".hr",".hsbc",".ht",".htc",".hu",".hughes",".hyatt",".hyundai",".ibm",".icbc",".anz",".ice",".icu",".id",".ie",".ieee",".ifm",".iinet",".ikano",".il",".im",".ao",".imamat",".imdb",".immo",".immobilien",".in",".industries",".infiniti",".info",".ing",".ink",".aol",".institute",".insurance",".insure",".int",".intel",".international",".intuit",".investments",".io",".ipiranga",".apartments",".iq",".ir",".irish",".is",".iselect",".ismaili",".ist",".istanbul",".it",".itau",".app",".itv",".iveco",".iwc",".jaguar",".java",".jcb",".jcp",".je",".jeep",".jetzt",".apple",".jewelry",".jio",".jlc",".jll",".jm",".jmp",".jnj",".jo",".jobs",".joburg",".aq",".jot",".joy",".jp",".jpmorgan",".jprs",".juegos",".juniper",".kaufen",".kddi",".ke",".aquarelle",".kerryhotels",".kerrylogistics",".kerryproperties",".kfh",".kg",".kh",".ki",".kia",".kim",".kinder",".ar",".kindle",".kitchen",".kiwi",".km",".kn",".koeln",".komatsu",".kosher",".kp",".kpmg",".abc",".arab",".kpn",".kr",".krd",".kred",".kuokgroup",".kw",".ky",".kyoto",".kz",".la",".aramco",".lacaixa",".ladbrokes",".lamborghini",".lamer",".lancaster",".lancia",".lancome",".land",".landrover",".lanxess",".archi",".lasalle",".lat",".latino",".latrobe",".law",".lawyer",".lb",".lc",".lds",".lease",".army",".leclerc",".lefrak",".legal",".lego",".lexus",".lgbt",".li",".liaison",".lidl",".life",".arpa",".lifeinsurance",".lifestyle",".lighting",".like",".lilly",".limited",".limo",".lincoln",".linde",".link",".art",".lipsy",".live",".living",".lixil",".lk",".llc",".loan",".loans",".locker",".locus",".arte",".loft",".lol",".london",".lotte",".lotto",".love",".lpl",".lplfinancial",".lr",".ls",".as",".lt",".ltd",".ltda",".lu",".lundbeck",".lupin",".luxe",".luxury",".lv",".ly",".asda",".ma",".macys",".madrid",".maif",".maison",".makeup",".man",".management",".mango",".map",".asia",".market",".marketing",".markets",".marriott",".marshalls",".maserati",".mattel",".mba",".mc",".mcd",".able",".associates",".mcdonalds",".mckinsey",".md",".me",".med",".media",".meet",".melbourne",".meme",".memorial",".at",".men",".menu",".meo",".merckmsd",".metlife",".mf",".mg",".mh",".miami",".microsoft",".athleta",".mil",".mini",".mint",".mit",".mitsubishi",".mk",".ml",".mlb",".mls",".mm",".attorney",".mma",".mn",".mo",".mobi",".mobile",".mobily",".moda",".moe",".moi",".mom",".au",".monash",".money",".monster",".montblanc",".mopar",".mormon",".mortgage",".moscow",".moto",".motorcycles",".auction",".mov",".movie",".movistar",".mp",".mq",".mr",".ms",".msd",".mt",".mtn",".audi",".mtpc",".mtr",".mu",".museum",".mutual",".mutuelle",".mv",".mw",".mx",".my",".audible",".mz",".na",".nab",".nadex",".nagoya",".name",".nationwide",".natura",".navy",".nba",".audio",".nc",".ne",".nec",".net",".netbank",".netflix",".network",".neustar",".new",".newholland",".auspost",".news",".next",".nextdirect",".nexus",".nf",".nfl",".ng",".ngo",".nhk",".ni",".abogado",".author",".nico",".nike",".nikon",".ninja",".nissan",".nissay",".nl",".no",".nokia",".northwesternmutual",".auto",".norton",".now",".nowruz",".nowtv",".np",".nr",".nra",".nrw",".ntt",".nu",".autos",".nyc",".nz",".obi",".observer",".off",".office",".okinawa",".olayan",".olayangroup",".oldnavy",".avianca",".ollo",".om",".omega",".one",".ong",".onl",".online",".onyourside",".ooo",".open",".aw",".oracle",".orange",".org",".organic",".orientexpress",".origins",".osaka",".otsuka",".ott",".ovh",".aws",".pa",".page",".pamperedchef",".panasonic",".panerai",".paris",".pars",".partners",".parts",".party",".ax",".passagens",".pay",".pccw",".pe",".pet",".pf",".pfizer",".pg",".ph",".pharmacy",".axa",".phd",".philips",".phone",".photo",".photography",".photos",".physio",".piaget",".pics",".pictet",".az",".pictures",".pid",".pin",".ping",".pink",".pioneer",".pizza",".pk",".pl",".place",".azure",".play",".playstation",".plumbing",".plus",".pm",".pn",".pnc",".pohl",".poker",".politie"]
    for l in domains:
        regex = re.compile(r'[0-9a-zA-Z_-]+' + l + '$')
        m = regex.findall(domain)
        if len(m) > 0:
            return m[0]
        else:
            pass
    return res

# generate url
def gen_urls(domain,ports):
    urls = []
    for i in ports:
        if i == '443':
            urls.append('https://%s' % domain)
        elif i == '80':
            urls.append('http://%s' % domain)
        else:
            urls.append('http://%s:%s' % (domain,i))
            urls.append('https://%s:%s' % (domain,i))
    return urls

# url filter
def url_filter(output_path,urls):
    u = []
    url_headers = {}
    f = open(output_path,'a+')
    for i in urls:
        try:
            r = requests.get(i,verify=False,timeout=(5,10),allow_redirects=False)
            f.write(i + '\n')
            u.append(i)
            h = r.headers
            h['status_code'] = r.status_code
            url_headers[i] = h
        except:
            pass
    f.close()
    return u,url_headers